import org.apache.spark.SparkContext
import org.apache.spark.ml.feature.{IDF, IDFModel}
import org.apache.spark.sql.{DataFrame, SparkSession}

object test3 {
  def main(args: Array[String]): Unit = {
    //1 创建sparkSession
    val spark: SparkSession = SparkSession.builder().appName("lris").master("local[*]").getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")

    //2 读取libsvm数据
    val lrisLibSvmDF: DataFrame = spark.read.format("libsvm")
      .load("file:///D:\\大数据\\学期文档\\项目\\03挖掘型标签\\数据集\\iris_kmeans.txt")

    //3 使用IDF来对单纯的词频特征向量进行修正
    val idf: IDF = new IDF().setInputCol("features").setOutputCol("new_features")
    val iDFModel: IDFModel = idf.fit(lrisLibSvmDF)

    val rescaledData: DataFrame = iDFModel.transform(lrisLibSvmDF)
    rescaledData.select("label","new_features").take(3).foreach(println)


  }
}
